#!/usr/bin/env python3 

import errno
import os
import sys
import xml.etree.ElementTree as ET
import requests
import rdflib
import hashlib

# Apache maintains this list of DOAP files for all Apache projects
# rdfpath = "https://svn.apache.org/repos/asf/comdev/projects.apache.org/data/projects.xml"
rdfpath = "https://svn.apache.org/repos/asf/comdev/projects.apache.org/trunk/data/projects.xml"

# Download list of projects and parse XML
try:
    projects_list = ET.fromstring (requests.get(rdfpath).text)
except Exception as e:
    print("Cannot download list")
    print(e)
    exit()

for project in projects_list: # Loop each element in the XML
    try:
        
        if project.tag != "location":
            continue
        
        # project.text is the URL of a project's DOAP file. Download that
        # file into a string
        doap_request = requests.get (project.text)
        
        if doap_request.status_code == 404:
            continue
        
        rdfxml_content = doap_request.text
        
        output_file = "data/" + hashlib.md5 (project.text.encode ("UTF-8")).hexdigest () + ".rdf"
        
        # Write the output we've just created
        with open (output_file, 'w') as rdf_file:
            rdf_file.write (rdfxml_content)
        
    except Exception as e:
        
        print ("Cannot process RDF file: ", project.text)
        print(e)
